package com.spider.silence.downloader;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.spider.silence.abstarct.downloader.AbstractLoginDownLoader;
import com.spider.silence.task.SinaSiChuanTask;
import com.spider.silence.task.manager.CommonTaskManager;
import com.spider.silence.utils.*;
import org.apache.log4j.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.openqa.selenium.Cookie;
import org.openqa.selenium.chrome.ChromeDriver;
import us.codecraft.webmagic.Page;

import java.io.File;
import java.lang.reflect.InvocationTargetException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;

/**
 * Created by Linbo Hu on 2017/6/20.
 * 新浪四川网爬虫下载器
 */
public class SinaSiChuanDownLoader extends AbstractLoginDownLoader implements Cloneable{

    private Logger logger =Logger.getLogger(ZuiChengduDownLoader.class);

    private final String[] section = {
            "http://interface.sina.cn/dfz/jx/news/index.d.html?callback=jsonp1498015510181&page=1&ch=zhengwen&cid=57610", //成都
            "http://interface.sina.cn/dfz/jx/news/index.d.html?callback=jsonp1498015528894&page=1&ch=zhengwen&cid=57611", //各地
            "http://interface.sina.cn/dfz/jx/news/index.d.html?callback=jsonp1498015550458&page=1&ch=zhengwen&cid=57612", //社会
            "http://interface.sina.cn/dfz/jx/news/index.d.html?callback=jsonp1498015575965&page=1&ch=zhengwen&cid=57620", //综合资讯
            "http://interface.sina.cn/dfz/jx/news/index.d.html?callback=jsonp1498015610153&page=1&ch=zhengwen&cid=57622", //热点评论
            "http://interface.sina.cn/dfz/jx/news/index.d.html?callback=jsonp1498015627169&page=1&ch=zhengwen&cid=57621", //小浪调查
            //"http://interface.sina.cn/dfz/jx/news/index.d.html?callback=jsonp1498015650351&page=1&ch=zhengwen&cid=57624", //锐评论
            "http://interface.sina.cn/dfz/jx/news/index.d.html?callback=jsonp1498015698489&page=1&ch=zhengwen&cid=57630", //体娱
            //"http://interface.sina.cn/dfz/jx/news/index.d.html?callback=jsonp1498015150054&ch=waterfall&cid=71022&page=1", //图片
            "http://interface.sina.cn/dfz/jx/news/index.d.html?callback=jsonp1498015386826&page=1&ch=zhengwen&cid=57614", //金融动态
            "http://interface.sina.cn/dfz/jx/news/index.d.html?callback=jsonp1498015425587&page=1&ch=zhengwen&cid=57623", //城市家园
            "http://interface.sina.cn/dfz/jx/news/index.d.html?callback=jsonp1498015467788&page=1&ch=zhengwen&cid=57629" //一本爱心
    };

    private final int pageSize = 200;

    //查看评论的url前缀
    private final String first = "http://comment5.news.sina.com.cn/page/info?format=js&channel=sic&newsid=";

    private final String second = "&group=&compress=1&ie=gbk&oe=gbk&page=";

    private final String last = "&page_size=200";

    private Map<String,File> files = null;

    public Map<String,File> getFiles(){
        return files;
    }

    @Override
    protected HashSet<Cookie> login() {
        return new HashSet<>();
    }

    //将cookies和headers克隆到每一个下载器中
    public SinaSiChuanDownLoader cloneDownLoader() throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
        SinaSiChuanDownLoader sinaSiChuanDownLoader = new SinaSiChuanDownLoader();
        sinaSiChuanDownLoader.cookies = (HashSet<Cookie>) this.cookies.clone();
        sinaSiChuanDownLoader.headers = (HashMap<String, String>) this.headers.clone();
        return sinaSiChuanDownLoader;
    }

    //生成爬虫任务
    public void generateTask(){
        for(int i = 0; i < section.length; i++) {
            try {
                String result = get(section[i]).getElementsByTag("body").text();
                result = result.substring(result.indexOf("(")+1, result.lastIndexOf(")")).trim();
                //result = result.replace("\"\"","\\\"\"").replace("\"\\", "\\\"\\").replace(":\\\"\\", ":\"\\");
                //System.out.println(result);
                JSONObject obj = JSONObject.parseObject(result);
                JSONArray list = ((JSONObject)((JSONObject)(obj.get("result"))).get("data")).getJSONArray("list");
                for(int j = 0; j < list.size(); j++) {
                    JSONObject jsonObject = (JSONObject) list.get(j);
                    String url = (String)jsonObject.get("URL");
                    SinaSiChuanTask task =new SinaSiChuanTask(url);
                    CommonTaskManager.add("SinaSiChuan", task);
                }
            } catch (Exception e) {
                e.printStackTrace();
                logger.error(e.getMessage(),e);
                continue;
            }
        }
    }


    public JSONObject crawlItem(String url) throws Exception {
        JSONObject item = new JSONObject();
        JSONObject data;
        //该字段用来标识该爬虫是否成功完成
        boolean success = false;
        long startTime = System.currentTimeMillis();
        long nowTime = startTime;
        while (nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("SinaSiChuan.timeOut"))) {
            //假如有异常, 就重新爬取，并将以前的数据重置
            data = new JSONObject();
            data.put("url", url);
            nowTime = System.currentTimeMillis();
            try {
                Page page = getHtml(url,true);
                if(page != null && page.getStatusCode() == 200) {
                    Document document = page.getHtml().getDocument();
                    String title = document.title();
                    //标题
                    data.put("title", title);
                    String publish_time = document.select("div[id=\"artibody\"] div[class=\"article-header clearfix\"] > p > span").get(0).text();
                    publish_time = publish_time.replace(" ", " ")+":00";
                    //新闻发布时间
                    data.put("publish_time", publish_time);
                    //爬取时间
                    data.put("spider_time", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()));
                    //假如存在图片, 则爬取图片
                    Elements imgElements = document.select("div[id =\"artibody\"] > div[class=\"article-body main-body\"] > div[class=\"img_wrapper\"]");
                    //存取新闻图片
                    List<String> img_urls = new ArrayList<>();
                    //抓取第一页图片
                    if(imgElements.size() > 0) {
                        for(int i = 0; i < imgElements.size(); i++) {
                            String img = imgElements.get(i).select("img").get(0).attr("src");
                            img_urls.add(img);
                        }
                    }

                    //如果有高清图片就抓取高清图片
                    Elements gaoqingElements = document.select("div[id =\"HdFigure1_1_0\"]");
                    if(gaoqingElements.size() > 0) {
                        //查找到高清图的url
                        Elements scriptElements = document.select("div[id =\"artibody\"] > div[class=\"article-body main-body\"] > script");
                        String script = scriptElements.get(0).toString();
                        //高清图url
                        script = script.substring(script.indexOf("http"), script.lastIndexOf(".html")+5);
                        Page gaoqingPage = getHtml(script, true);
                        if(gaoqingPage != null && gaoqingPage.getStatusCode() == 200) {
                            Document gaoqingDocument = gaoqingPage.getHtml().getDocument();
                            //高清图片的dl标签
                            Elements gaoqingtupianElements = gaoqingDocument.select("div[id =\"eData\"] > dl");
                            for(int i = 0; i < gaoqingtupianElements.size(); i++) {
                                img_urls.add(gaoqingtupianElements.get(i).select("dd").get(0).text());
                            }
                        }
                    }

                    //视频组件
                    Elements videoElements = document.select("div[id =\"artibody\"] > div[class=\"article-body main-body\"] > div[class=\"artical-player-wrap\"]");
                    if(videoElements.size() > 0) {
                        List<String> videoList = new ArrayList<>();
                        Element videoElement = videoElements.get(0);
                        String video_url = videoElement.select("script").get(6).toString();
                        video_url = video_url.substring(video_url.indexOf("http"), video_url.lastIndexOf(".html")+5);
                        videoList.add(video_url);
                        data.put("video_urls", videoList.toArray());
                    }

                    //加载第一页内容
                    Elements contentElements = document.select("div[id=\"artibody\"] > div[class=\"article-body main-body\"] > p");
                    StringBuilder content = new StringBuilder();
                    for(int i = 0; i < contentElements.size(); i++) {
                        content.append(contentElements.get(i).text());
                    }
                    //设置html文件的命名格式
                    String fileName = "SinaSiChuan" + "_" + DateUtils.dateToTimestamp( DateUtils.parseDate(publish_time)).getTime() + "_" + MD5Utils.getHash3(url.replaceAll("#.*", ""), "MD5");
                    String publishDate = DateUtils.formatDate(DateUtils.parseDate(publish_time)).replace("-","-");
                    String all_fileName = "新浪四川_speeches_" + publishDate + "_" + fileName;
                    //抓取第一页的网页源码
                    File file = FileUtils.createTempFile(all_fileName + "_pageNo_1", document.html().toString());
                    files = new ConcurrentHashMap<>();
                    files.put(all_fileName + "_pageNo_1", file);

                    //如果有多页, 就爬取后面的几页
                    Elements pageElements = document.select("div[id=\"artibody\"] > div[class=\"article-body main-body\"] > div[class=\"page\"] > a");
                    if(pageElements.size() > 0) {
                        for(int i = 0; i < pageElements.size(); i++) {
                            if(pageElements.get(i).text().equals("下一页")) {
                                continue;
                            }
                            String pageUrl = pageElements.get(i).attr("href");
                            Page later_page = getHtml(pageUrl,true); //后面的页面
                            //后面页面的内容
                            Elements later_content = later_page.getHtml().getDocument().select("div[id=\"artibody\"] > div[class=\"article-body main-body\"] > p");
                            if(later_content.size() > 0) {
                                for(int j = 0; j < later_content.size(); j++) {
                                    content.append(later_content.get(j).text());
                                }
                            }
                            Elements later_img = later_page.getHtml().getDocument()
                                    .select("div[id =\"artibody\"] > div[class=\"article-body main-body\"] > div[class=\"img_wrapper\"]");
                            //图片
                            if(later_img.size() > 0) {
                                for(int j = 0; j < later_img.size(); j++) {
                                    String img = later_img.get(j).select("img").get(0).attr("src");
                                    img_urls.add(img);
                                }
                            }
                            //抓取源码
                            File later_file = FileUtils.createTempFile(all_fileName + "_pageNo_" + (2+i), later_page.getHtml().getDocument().html().toString());
                            files.put(all_fileName + "_pageNo_" + (i+2), later_file);
                        }
                    }
                    data.put("img_urls", img_urls.toArray());
                    //内容
                    data.put("content", content);
                    //查看评论
                    String newid = (((document.select("script").get(11).toString()).split(";")[3]).replace("\"", "").trim()).split("=")[1];
                    data.put("id", newid);

                    //默认加载第一页的评论
                    String comment_url = first + newid + second + 1 + last;
                    //该新闻评论的json数据
                    Page comment_page = getHtml(comment_url, true);
                    if(comment_page == null) {
                        continue;
                    }
                    String result = (comment_page.getHtml().getDocument().getElementsByTag("body").text().substring(9)).trim();
                    JSONObject object = JSONObject.parseObject(result);
                    int reply_count = (Integer) ((JSONObject)((JSONObject) object.get("result")).get("count")).get("show");
                    //回复数(评论数)
                    data.put("read_count", reply_count);
                    //阅读量(参与数)
                    data.put("reply_count", ((JSONObject)((JSONObject) object.get("result")).get("count")).get("total"));
                    //新闻回复
                    JSONArray reply_array = new JSONArray();
                    //第一页的评论
                    JSONArray first_page_comment_array = ((JSONObject)object.get("result")).getJSONArray("cmntlist");
                    if(first_page_comment_array.size() > 0) {
                        for(int i = 0; i < first_page_comment_array.size(); i++) {
                            //原始的json
                            JSONObject original = (JSONObject)first_page_comment_array.get(i);
                            //要存入的json
                            JSONObject temp = new JSONObject();
                            temp.put("like_count", original.get("agree"));
                            temp.put("dislike_count", original.get("against"));
                            temp.put("content", original.get("content"));
                            String nick = ((String)original.get("config")).trim();
                            if(nick.indexOf("wb_screen_name=") > 0) {
                                nick = nick.substring(nick.indexOf("wb_screen_name=")+15);
                            }
                            if(nick.indexOf("&") > 0) {
                                nick  = nick.substring(0, nick.indexOf("&"));
                            }
                            temp.put("publish_user",nick);
                            temp.put("publish_user_id", original.get("uid"));
                            temp.put("publish_time", original.get("time"));
                            temp.put("ancestor_id", original.get("newsid"));
                            String img = ((String)original.get("config")).trim();
                            int begin = img.indexOf("http");
                            if(begin > 0) {
                                img = img.substring(begin);
                                if(img.indexOf("&") > 0) {
                                    img  = img.substring(0, img.indexOf("&"));
                                }
                            }
                            img = img.replace("%3A%2F%2F", "://").replace("%2F", "/").trim();
                            temp.put("publish_user_photo", img);
                            reply_array.add(temp);
                        }
                    }
                    int page_count; //新闻回复的页码数
                    if(reply_count % 200 == 0) {
                        page_count = reply_count / 200;
                    } else {
                        page_count = reply_count / 200 + 1;
                    }
                    //如果还有后面的页码,则继续爬取后面页码的评论
                    if(page_count > 1) {
                        //爬取后面几页的评论
                        for(int i = 2; i < page_count + 1; i++) {
                            String later_url = first + newid + second + i + last;
                            //后面几页新闻评论的json数据
                            Page later_page = getHtml(later_url, true);
                            if(comment_page == null) {
                                continue;
                            }
                            String later_result = (later_page.getHtml().getDocument().getElementsByTag("body").text().substring(9)).trim();
                            JSONObject later_object = JSONObject.parseObject(later_result);
                            JSONArray later_page_comment_array = ((JSONObject)later_object.get("result")).getJSONArray("cmntlist");
                            if(later_page_comment_array.size() > 0) {
                                for(int j = 0; j < later_page_comment_array.size(); j++) {
                                    //原始的json
                                    JSONObject original = (JSONObject)later_page_comment_array.get(j);
                                    //要存入的json
                                    JSONObject temp = new JSONObject();
                                    temp.put("like_count", original.get("agree"));
                                    temp.put("dislike_count", original.get("against"));
                                    temp.put("content", original.get("content"));
                                    temp.put("publish_user", original.get("nick"));
                                    temp.put("publish_user_id", original.get("uid"));
                                    temp.put("publish_time", original.get("time"));
                                    temp.put("ancestor_id", original.get("newsid"));
                                    temp.put("publish_user", original.get("nick"));
                                    String img = ((String)original.get("config")).trim();
                                    int begin = img.indexOf("http");
                                    int end = 0;
                                    int jpg_suffix = img.lastIndexOf(".jpg");
                                    int gif_suffix = img.lastIndexOf(".gif");
                                    if(jpg_suffix > 0) {
                                        end = jpg_suffix + 4;
                                    } else if(gif_suffix > 0) {
                                        end = gif_suffix + 4;
                                    }
                                    if(begin > 0 && end > 0) {
                                        img = img.substring(begin, end);
                                    }
                                    img = img.replace("%3A%2F%2F", "://").replace("%2F", "/").trim();
                                    temp.put("publish_user_photo", img);
                                    temp.put("publish_user_photo", img);
                                    reply_array.add(temp);
                                }
                            }
                        }
                    }
                    data.put("reply_nodes", reply_array);
                    item.put("data", data);
                    success = true;
                    break;
                }
            } catch (Exception e) {
                //e.printStackTrace();
                //等三秒继续爬
                Thread.sleep(1500);
                logger.info(e);
                System.err.println("爬虫过程出现异常, 尝试重新爬取");
                continue;
            }
        }
        if(success == true) {
            return item;
        } else {
            throw new Exception(url+"爬取失败");
        }
    }
}
